package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
)

func main() {
	var start, end int
	fmt.Println("请输入爬取起始页（>=1）：")
	fmt.Scan(&start)
	fmt.Println("请输入爬取终止页(>=start);")
	fmt.Scan(&end)
	toWork(start, end)
}

func toWork(start int, end int) {
	fmt.Printf("正在爬取%d页到%d页的数据\n", start, end)
	for i := start; i <= end; i++ {
		SpiderPage2(i)
	}
}

func SpiderPage2(idx int) {
	//获取url
	url := "https://movie.douban.com/top250?start=" + strconv.Itoa((idx-1)*25) + "&filter="

	//封装HttpHGet2爬取url对应的页面
	result, err := HttpGet2(url)
	if err != nil {
		fmt.Println("Get2 err", err)
		return
	}
	//fmt.Println("result",result)

	//解析、编译正则表达式——电影名称
	ret1 := regexp.MustCompile(`<img width="100" alt="(?s:(.*?))"`)
	//提取需要信息
	filmName := ret1.FindAllStringSubmatch(result, -1)
	//for _, name := range filmName {
	//	fmt.Println("name",name[1])
	//}

	//解析、编译正则表达式——分数
	ret2 := regexp.MustCompile(`<span class="rating_num" property="v:average">(?s:(.*?))</span>`)
	//提取需要信息
	filmScore := ret2.FindAllStringSubmatch(result, -1)
	//for _, score := range filmScore {
	//	fmt.Println("score",score[1])
	//}

	//解析、编译正则表达式——人数
	ret3 := regexp.MustCompile(`<span>(.*?)人评价</span>`) //去掉单行模式
	//提取需要信息
	peopleNum := ret3.FindAllStringSubmatch(result, -1)
	//for _, num := range peopleNum {
	//	fmt.Println("num",num[1])
	//}
	//将提取的有用信息，封装到文件中
	saveToFile(idx, filmName, filmScore, peopleNum)

}

//
func saveToFile(idx int, filmName, filmScore, peopleNum [][]string) {
	file, err := os.Create("第" + strconv.Itoa(idx) + "页.txt")
	if err != nil {
		fmt.Println("os.Create", err)
	}
	defer file.Close()

	n := len(filmName) //得到条目数

	//先打印抬头
	file.WriteString("电影名称" + "\t\t\t" + "评分" + "\t\t" + "评分人数\n")

	for i := 0; i < n; i++ {
		file.WriteString(filmName[i][1] + "\t\t\t" + filmScore[i][1] + "\t\t" + peopleNum[i][1] + "\n")
	}
}

//爬取制定url页面，返回b
func HttpGet2(url string) (result string, err error) {

	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal(err)
	}
	//req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36")
	req.Header.Set("User-Agent", "")

	resp, err1 := client.Do(req)

	//resp, err1 := http.Get(url)
	//fmt.Println("Header",resp.Header)
	if err1 != nil {
		err = err1
		return
	}
	defer resp.Body.Close()

	buf := make([]byte, 4096)

	//循环查询整页数据
	for {
		n, err2 := resp.Body.Read(buf)
		if n == 0 {
			fmt.Println("读取网页完成")
			break
		}
		if err2 != nil && err2 != io.EOF {
			err = err2
			return
		}
		result += string(buf[:n])
	}
	return
}
